package engine

type ConcurrentEngine struct {
	Scheduler Scheduler
	WorkerCount int    // 启动的协程数量
	ItemChan    chan Profile  // 用于开协程进行数据存储
}
type Scheduler interface {
	Submit(Request)   // 向Worker提交请求处理任务
	WorkerChan () chan Request
	ReadyNotify   // 当前Worker准备好了
	Run()  // 启动运行
}

type ReadyNotify interface {
	WorkerReady (chan Request)
}

/**
 * 队列分发式分布式爬虫调度器
 */
 func (e *ConcurrentEngine) Run(seeds ...Request) {
 	out := make(chan ParserResult)   // 任务输出channel
 	e.Scheduler.Run()

 	for i := 0; i < e.WorkerCount; i++ {
		createWorker(e.Scheduler.WorkerChan(), out, e.Scheduler)
	}
 	for _, r := range seeds {
 		e.Scheduler.Submit(r)
	}

 	for {
 		result := <- out
 		// 这里先打印结果，之后会进行后续处理
 		if result.Item.Title != "" {
			go func() {
				e.ItemChan <- result.Item
			}()
		}

 		for _, request := range result.Request {
 			if request.Url != "" {
				e.Scheduler.Submit(request)
			}
		}
	}
 }

 // 创建一个协程任务
 func createWorker(in chan Request, out chan ParserResult, ready ReadyNotify){
 	go func(){
 		for {
 			ready.WorkerReady(in)
 			request := <- in
 			result, err := worker(request)
 			if err != nil {
 				continue
			}
 			out <- result   // 将正确结果送出
		}
	}()
 }